package com.a.b.c.d.e.webmagic.processors;

import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ObjectUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Collections;
import java.util.List;

/**
 * Created on 2021/9/15.
 *
 * @author yanshuchun
 */
@Slf4j
public class BaiduPageProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    public static final String URL_LIST = "https://www.ygdy8.com/html/gndy/dyzz/list_\\d*_\\d*.html";
    public static final String URL_POST = "https://www.ygdy8.com/html/gndy/dyzz/\\d*/\\d*.html";

    //    .co_content8 > ul:nth-child(1)
    @Override
    public void process(Page page) {
        Selectable url = page.getUrl();
        Html html = page.getHtml();
        if (url.regex(URL_LIST).match()) {
            List<String> all = html.css(".co_content8 > ul:nth-child(1)").links().all();
            List<String> posturls = Collections.synchronizedList(all);
            posturls.parallelStream().forEach(item -> {
                if (item.matches(URL_POST)) {
                    page.addTargetRequest(item);
                    log.info("合规：{}", item);
                } else {
                    log.info("不合规：{}", item);
                }
            });
        }
        List<String> all = html.css(".co_content8 > div:nth-child(2)").links().all();
        Collections.synchronizedList(all).parallelStream().forEach(item -> {
            if (item.matches(URL_LIST)) {
                page.addTargetRequest(item);
            }
        });


        if (url.regex(URL_POST).match()) {
            Selectable name = html.css(".bd3l > div:nth-child(2) > div:nth-child(1) > h1:nth-child(1) > font:nth-child(1)", "text");

            Selectable downloadUrl = html.css("#Zoom > span:nth-child(1) > a", "href");

            if (ObjectUtils.isEmpty(name.get())) {
                name = html.css("div.co_area2:nth-child(2) > div:nth-child(1) > h1:nth-child(1) > font:nth-child(1)", "text");
                downloadUrl = html.css("#Zoom > span:nth-child(1) > table:nth-child(7) > tbody:nth-child(1) > tr:nth-child(1) > td:nth-child(1) > a", "href");
            }
            System.out.println("名称：" + name);
            System.out.println("下载地址：" + downloadUrl);


        }
    }

    @Override
    public Site getSite() {
        System.out.println("获取访问站点++++++++++++++++++++++++++++++++++++++++++++++++++++++++");
        return site;
    }
}
